# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import hashlib
import time
import pymysql
from MeiHaoTong.py_bloomfilter import PyBloomFilter


class MeihaotongPipeline(object):
    def __init__(self, host, database, user, password, port,content_table):
        self.host = host
        self.database = database
        self.user = user
        self.password = password
        self.port = int(port)
        self.content_table = content_table

    @classmethod
    def from_crawler(cls, crawler):
        MYSQL_SETTINGS = crawler.settings.get('MYSQL_SETTINGS')
        return cls(
            host=MYSQL_SETTINGS['MYSQL_HOST'],
            user=MYSQL_SETTINGS['MYSQL_USER'],
            password=MYSQL_SETTINGS['MYSQL_PASSWORD'],
            port=MYSQL_SETTINGS['MYSQL_PORT'],
            database=MYSQL_SETTINGS['MYSQL_DATABASE'],
            content_table=MYSQL_SETTINGS['CONTENT_TABLE']
        )

    def open_spider(self, spider):
        self.db = pymysql.connect(self.host, self.user, self.password, self.database, self.port, charset='utf8')
        self.cursor = self.db.cursor()

    def close_spider(self, spider):
        self.db.close()

    def process_item(self, item, spider):
        item['spider_time'] = time.strftime("%Y-%m-%d %X", time.localtime())
        finger = hashlib.sha1((item['title']+str(item['site_id'])).encode("utf-8")).hexdigest()
        bf = PyBloomFilter()
        if not bf.is_exist(finger): #如果不存在
            bf.add(finger)
            insert_sql = "insert into {}(title,url,browse_count,comment_count,site_id,author,site_classify_id,publish_time,spider_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)".format(self.content_table)
            self.cursor.execute(insert_sql, (
                item["title"], item["url"], item['browse_count'],item["comment_count"], item['site_id'],item['author'],item['site_classify_id'],item["publish_time"], item["spider_time"]))
            self.db.commit()
        return item
